# coding=utf-8
import requests
from lxml import etree


def Read_Url():
    with open('./urls.list', 'r', encoding='utf-8') as f:
        lines = f.readlines()
    for line in lines:
        yield line.strip('\n')


def Read_detail_url():
    for Url in Read_Url():
        text = requests.get(Url).text
        html = etree.HTML(text)
        url_list = html.xpath('//div[@class="listbox"]/ul[@class="e2"]/li/a/@href')
        for url in url_list:
            url = 'http://www.ukassignment.org' + url
            yield url


def Format_content(p_list):
    item = []
    for p in p_list:
        p1 = str(p).replace('\r\n', '')
        p2 = str(p1).replace('\t\xa0', '')
        p3 = str(p2).replace('\t', '')
        p4 = str(p3).replace('\xa0', '')
        p5 = p4.strip()
        if p5 != '':
            item.append(p5)
    if len(item) != 0:
        return item


def Save(title, item):
    if title is not None and item is not None:
        with open('./txt/{}.txt'.format(title), 'a+', encoding='utf-8') as f:
            f.write('<title={}>\n'.format(title))
        with open('./txt/{}.txt'.format(title), 'a+', encoding='utf-8') as f:
            f.write('<neirong=\n')
        for p in item:
            with open('./txt/{}.txt'.format(title), 'a+', encoding='utf-8') as f:
                f.write('<p>{}</p>\n'.format(p))
        with open('./txt/{}.txt'.format(title), 'a+', encoding='utf-8') as f:
            f.write('>')


def Format_title(title):
    t1 = str(title).replace('\\', '')
    t2 = str(t1).replace('/', '')
    t3 = str(t2).replace(':', '：')
    t4 = str(t3).replace('*', '')
    t5 = str(t4).replace('?', '？')
    t6 = str(t5).replace('<', '《')
    t7 = str(t6).replace('>', '》')
    t8 = str(t7).replace('|', '')
    t9 = str(t8).replace('"', '”')
    return t9.strip()


# 请求网页，解析网页内容
def run():
    for url in Read_detail_url():
        try:
            response = requests.get(url)
            response.encoding = 'utf-8'
            html = etree.HTML(response.text)
            title_list = html.xpath('//div[@class="title"]/h1/text()')
            title = title_list[0] if len(title_list) != 0 else None
            # print(title)
            if title is not None:
                p_list = html.xpath('//div[@class="content"]//text()')
                item = Format_content(p_list)
                title = Format_title(title)
                Save(title, item)
                print(title, '写入成功......')
            else:
                return None, None
        except:
            pass


if __name__ == '__main__':
    run()
